\documentclass[twoside,a4paper]{article}
\usepackage{geometry}
\geometry{margin=1.5cm, vmargin={0pt,1cm}}
\setlength{\topmargin}{-1cm}
\setlength{\paperheight}{29.7cm}
\setlength{\textheight}{25.3cm}

% useful packages.
\usepackage{amsfonts}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{enumerate}
\usepackage{graphicx}
\usepackage{multicol}
\usepackage{fancyhdr}
\usepackage{layout}

\begin{document}
	
	\pagestyle{fancy}
	\fancyhead{}
	\lhead{Haolong Li}
	\chead{ DMAA Homework \#3}
	\rhead{\today}
	

\section*{6.1. Write the gradient descent algorithm of logistic regression model}

Solution:
	From the textbook page 93, we know the logarithmic likelihood function of logistic regression moder is 
	\[		
		L(w)=\sum_{i=1}^{N}[y_{i}\log \pi (x_{i})+(1-y_{i}) \log (1- \pi (x_{i}))] 
		=\sum_{i=1}^{N}[y_{i} \log \frac{\pi (x_{i})}{1- \pi (x_{i})}+ \log (1- \pi (x_{i}))] 
		=\sum_{i=1}^{N}[y_{i}(w \cdot x_{i})- \log (1+ \exp (w \cdot x_{i}))]		
	\]
	
	Take a derivative of w, we have
	
	\[
		\frac{\partial L(w)}{\partial w} = \sum_{i=1}^{N}[x_{i} \cdot y_{i}- \frac{\exp (w \cdot x_{i} \cdot x_{i})}{1 + \exp (w \cdot x_{i})}]				
	\]
$ \Rightarrow $

\[
	\nabla L(w)=[\frac{\partial L(w)}{\partial w(0)}, \cdots, \frac{\partial L(w)}{\partial w(n)}]
\]

Following gradient descent algorithm, the algorithm is showed below:

Input: $L(w),\epsilon$

Output: $w^{*},k$

(1) Set a initial value of w: $w_{0} \in R^{n}$, set compute step k = 0, go (2);

(2) Compute $L(w_{k})$, go (3);

(3) Set kth step's search direction $p_{k}=\nabla L(w_{k})$, if $\Vert p_{k} \Vert < \epsilon$, go (6), else, go (4);

(4) Find $\alpha _{k}$ that maxminize $L(w_{k}+ \alpha_{k} p_{k} )$, and set $w_{k+1}=w_{k}+\alpha_{k} p_{k}$

(5) If $\Vert L(w_{k+1}) - L(w_{k}) \Vert < \epsilon$, let $w^{*} = w_{k+1}$, go (6);

(6) Output $w^{*}$, stop the loop.

\end{document}

%%% Local Variables: 
%%% mode: latex
%%% TeX-master: t
%%% End: 
